# -*- coding: utf-8 -*-

import random
import datetime

BOT_NAME = 'HifoEsf'

SPIDER_MODULES = ['HifoEsf.spiders']
NEWSPIDER_MODULE = 'HifoEsf.spiders'

# 指定request对象的去重过滤器
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 指定scheduler队列 (url调度器队列)
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# url调度器持久化
SCHEDULER_PERSIST = True
# 是否在开始之前清空 调度器和去重记录，True=清空，False=不清空，默认False
# SCHEDULER_FLUSH_ON_START = False

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'HifoEsf (+http://www.yourdomain.com)'

# Obey robots.txt rules
# ROBOTSTXT_OBEY = True
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = random.uniform(1.5, 2.0)
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
# }

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'HifoEsf.middlewares.HifoesfSpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
    "HifoEsf.middlewares.CustomRetryMiddleware": 500,

    'HifoEsf.middlewares.UserAgentMiddleware': 544,
    'HifoEsf.middlewares.OuterNetProxyMiddleware': 545,  # 启用redis中的代理，需要同时开启 RedisConnPipeline
}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
EXTENSIONS = {
    # 'scrapy.extensions.telnet.TelnetConsole': None,
    'HifoEsf.extentions.CustomRedisSpiderIdleClosedExtensions': 500,
}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'HifoEsf.pipelines.RedisConnPipeline': 299,  # 启用redis中的代理，需要同时开启 XXXProxyMiddleware
    'HifoEsf.pipelines.PcdPipeline': 300,  # 用于省市区id处理
    'HifoEsf.pipelines.FangduoduoPipeline': 301,
    'HifoEsf.pipelines.BiekeCjPipeline': 302,

    'HifoEsf.pipelines.MongoClientPipeline': 399,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
AUTOTHROTTLE_ENABLED = True
# The initial download delay
AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# 日志
today = datetime.datetime.now()
LOG_PATH = "{}/logs/{}_{}_{}.log".format(BOT_NAME, today.year, today.month, today.day)
# LOG_FILE = LOG_PATH
# LOG_LEVEL = "WARNING"
LOG_LEVEL = "DEBUG"

# UA池
USER_AGENTS = [
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)',
]

# 开启自定义扩展
EXT_ENABLED = True
# 配置允许的空闲时长
# IDLE_NUM*5  秒后，爬虫程序关闭
IDLE_NUM = 120

# redis(方式一)
# redis://[:password]@host:port/db
# rediss://[:password]@host:port/db
REDIS_URL = "redis://127.0.0.1:6379/6"
# redis(方式二)
# REDIS_HOST = "127.0.0.1"
# REDIS_PORT = 6379
# REDIS_PASSWORD = ''

# 重试的 response 返回值(code)
RETRY_HTTP_CODES = [500, 502, 503, 504, 400, 404, 408, 407, 418]

# 下载超时
DOWNLOAD_TIMEOUT = 10

# MONGODB
# MONGO_URI = 'mongodb://user:password@127.0.0.1:27017'
"""
1.replicaSet的name为空则不使用集群配置
2.user和pwd为空则不需要进行密码校验
"""
MONGODB = {
    'host': '192.168.5.236',
    'port': '27017',
    'user': '',
    'pwd': '',
    'replicaSet': {
        'name': True,
        "members": [
            {
                "host": "192.168.5.150",
                "port": "27017"
            },
            {
                "host": "192.168.5.151",
                "port": "27017"
            },
            # {
            #     "host": "192.168.5.163",
            #     "port": "27017"
            # },
        ]
    }
}

# 代理API
PROXY_URL = 'http://183.230.7.247:10888/getip?addr=&tof=f'

# 爬取城市配置
# 需要爬取的城市，优先度高于FILTER_CITIES
CRAWL_CITIES = ['重庆', '北京', '上海', '厦门', '武汉']
# 需要过滤的城市
# FILTER_CITIES = ['阿克苏']
